// Education data types with efficient storage and retrieval
//
// Note: If you call a constructor with a single numeric argument,
// it will preallocate that many array elements, rather than use the argument
// as a single data value. (This is the Array constructor's behavior.)

var edDataTypes = [];  // maintain a list of data type constructors

function EdDatum(values) {
    // Construct a generic education data type.
    // Interpret values argument(s) as arguments to Array().
    // Keep data as array in property v.
    // If values is already an eddata type, copy its data.
    if (arguments.length == 1 && values instanceof EdDatum) {
        this.v = new Array(values.v.length);
        for (var i = 0; i < values.v.length; i++)
            this.v[i] = values.v[i];
    } else {
        this.v = Array.apply(new Array, arguments);
    }
    this.type = arguments.callee;
}
EdDatum.prototype = {
    prefer:  'string',  // prefer 'string' or 'number' values
    precision:  null,  
        // n-digit strings like organization codes,
        // or numbers to display n digits to the right of the decimal
    label:  'ed data',
    description:  '',
    validExp:  null,    // override with regular expression
    validRange:  { min: null, max: null, integer: null },
    validNull:  true,   // if missing values are valid
    validSet:  [],      // override with a complete set of valid values
    case:  null,        // 'upper', 'lower', or null --> don't care
    valid:  edDatumValid,
    fix:  edDatumFix,
    scanValues:  edDatumScan,
    // scanValues() sets the following properties.
    uniqCount:  null,
    hasMissing:  null,
    uniqNumCount:  null,
    uniqTextCount:  null,
    min:  null,
    max:  null,
    uniqValues:  [],    // example values
    maxUniqValuesToKeep: 30,
    mostlyNum:  null
};
edDataTypes.push(EdDatum);

function edDatumValid(i) {
    // Return true if value at index i is valid.
    // Apply regular expression, min/max/integer, and missing value checks.
    // If no value has been defined at index i, return false;
    var v = this.v[i];
    if (v === undefined) return false;
    if (v == null) return this.validNull;
    if (this.prefer == 'number' && (v === '' || isNaN(Number(v)))) return false;
    if (this.validSet.length) return (this.validSet.indexOf(v) != -1);
    var match = this.validExp == null || this.validExp.test(v);
    var inRange = this.validRange == null 
        ||     (this.validRange.min == null || v >= this.validRange.min) 
        && (this.validRange.max == null || v <= this.validRange.max)
        && (this.validRange.integer == null || (v - Math.floor(v) == 0) == this.validRange.integer);
    return match && inRange;
}

function edDatumFix(i) {
    // Try to fix the data value at index i.
    // Try to store values in accordance with this.prefer, this.precision, and this.case.
    // For numeric variables, translate empty strings and "NA" to null values.
    // Return 
    //   null --> not implemented
    //    0   --> value valid, no fix necessary
    //    1   --> successful fix
    //   -1   --> unsuccessful fix attempt
    var digitsExp = /\d+/;  // decimal digits
    var missingNumValExp = /^na$|^-$|^$/i;
    var v = this.v[i];
    var saved = v;
    var fixed = 0;  // return this if we didn't change an already valid value
    switch (this.prefer) {
        case 'string':
            if (typeof v != 'string') {
                v = String(v);
                fixed = (v == '[object Object]') ? -1 : 1;
            }
            if (fixed >= 0 && this.precision && digitsExp.test(v)) {
                // Pad with leading zeroes as necessary.
                while (v.length < this.precision)
                    v = '0' + v;
                fixed = 1;
            }
            if (fixed >= 0 && this.case) {
                var vUncased = v;
                switch (this.case) {
                    case 'upper':
                        v = v.toUpperCase();
                        break;
                    case 'lower':
                        v = v.toLowerCase();
                        break;
                }
                if (v != vUncased) fixed = 1;
            }
            break;
        case 'number':
            if (typeof v != 'number') {
                if (missingNumValExp.test(v)) {
                    v = null;
                    fixed = 1;
                }
                else if (v !== null) {
                    v = Number(v);
                    fixed = isNaN(v) ? -1 : 1;
                }
            }
            break;
        default:
            return null;  // not implemented for this.prefer type
    }
    if (fixed > 0)
        this.v[i] = v;  // store fixed value
    if (!this.valid(i)) {
        this.v[i] = saved;  // replace original value
        fixed = this.valid[i] ? 0 : -1;  // if it wasn't broke we didn't fix it
    }
    return fixed;
}

function edDatumScan() {
    this.uniqCount = 0;
    this.hasMissing = false;
    this.uniqNumCount = 0;
    this.uniqTextCount = 0;
    this.min = null;
    this.max = null;
    this.uniqValues = [];
    var u = pv.uniq(this.v);
    u.forEach(edDatumScan1, this);
    this.mostlyNum = this.uniqNumCount > this.uniqTextCount * 4;
        // true if range of values is predominantly numeric
}

function edDatumScan1(val) {
    this.uniqCount++;
    if (val === null)
        this.hasMissing = true;
    else if (typeof val == 'string') {
        this.uniqTextCount++;
        if (this.uniqValues.length < this.maxUniqValuesToKeep)
            this.uniqValues.push(val);
    }
    else {
        this.uniqNumCount++;
        if (this.min === null || val < this.min)
            this.min = val;
        if (this.max === null || val > this.max)
            this.max = val;
    }
}

function MAOrgCode(codes) {
    // Construct MA organization ID, a string of eight digits, often with leading zeros.
    // Disallow missing values.
    // Fix values by adding leading zeros.
    EdDatum.apply(this, arguments);  // construct EdDatum first
    this.type = arguments.callee;
    this.label = 'MA organization code';
    this.prefer = 'string';
    this.precision = 8;  // eight digits
    this.validExp = /^\d{8}$/;  // eight digits
    this.validNull = false;
}
MAOrgCode.prototype = new EdDatum;
edDataTypes.push(MAOrgCode);

function StudentID(ids) {
    // Student ID, stored as a string.
    // Value required.
    EdDatum.apply(this, arguments);  // construct EdDatum first
    this.type = arguments.callee;
    this.label = 'student ID';
    this.validNull = false;
}
StudentID.prototype = new EdDatum;
edDataTypes.push(StudentID);

function MASASID(sasids) {
    // MA SASID, state assigned student identifier, 10 digits.
    // Value required.
    StudentID.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'SASID';
    this.precision = 10;
    this.validExp = /^\d{10}$/;  // ten digits
}
MASASID.prototype = new StudentID;
edDataTypes.push(MASASID);

function Name(value) {
    // A person's name.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'name';
}
Name.prototype = new EdDatum;
edDataTypes.push(Name);

function Teacher(value) {
    Name.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'teacher';
}
Teacher.prototype = new Name;
edDataTypes.push(Teacher);

function yearFix(i) {
    var v, saved;
    var fixed = edDatumFix.call(this, i);  // carry out generic fixes first
    if (fixed == -1) {
        v = saved = this.v[i];
        v = Number(v);
        if (v < 100) {
            v += 2000;
            this.v[i] = v;
            if (edDatumFix.call(this, i) > -1)  // check against range
                fixed = 1;
            // otherwise fixed is still = -1
        }
    }
    return fixed;
}

function CalendarYear(years) {
    // As opposed to SchoolYear.
    // Valid: integers between 1900 and 3000.
    // Require values.
    // Fix values less than 100 by adding 2000.
    // The asDate method returns a Javascript date 
    //   on a given m/d or Jan 1 of the year.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.prefer = 'number';
    this.label = 'calendar year';
    this.validRange.min = 1900;
    this.validRange.max = 3000;
    this.validRange.integer = true;
    this.validNull = false;
    this.fix = yearFix;
    this.asDate = 
        function(i, mo, da) {
            !mo && (mo = 1);
            !da && (da = 1);
            return new Date(this.v[i], mo, da);
        };
}
CalendarYear.prototype = new EdDatum;
edDataTypes.push(CalendarYear);

function schoolYearFix(i) {
    var v, saved;
    var fixed = yearFix.call(this, i);  // carry out calendar year fixes first
    if (fixed >= 0)
        return fixed;
    else {
        v = saved = this.v[i];
        v = String(v);
        var commonExp = /(\d{2,4})\s*(-|\/| )\s*(\d{2,4})/;  // 09-10, 2009-10, 2009-2010
        var match = commonExp.exec(v);
        if (match) {  // interpret common format
            var year1 = Number(match[1]);
            var year2 = Number(match[3]);
            if (year1 < 100) year1 += 2000;
            if (year2 < 100) year2 += 2000;
            if (year2 = year1 + 1) {
                v = year2;
                this.v[i] = v;
                if (yearFix.call(this, i) > -1)  // recheck against range
                    fixed = 1;  // fixed it
                else {
                    // fixed still = -1
                    this.v[i] = saved;  // restore original value
                }
            }
        }
        return fixed;
    }
}

function SchoolYear(years) {
    // School years, like financial years, are denoted by their ending year.
    // For example, the 2010-2011 school year is denoted by 2011.
    // The fix method attempts to convert strings like '2010-11' to school years.
    // The commonFormat method generates a 'yyyy-yyyy' string.
    CalendarYear.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'school ending year';
    this.fix = schoolYearFix;
    this.commonFormat =
        function (i) {
            if (this.valid(i))
                return (this.v[i] - 1) + '-' + this.v[i];
            else
                return this.v[i];
        };
}
SchoolYear.prototype = new CalendarYear;
edDataTypes.push(SchoolYear);

var gradeStrings = [
    'PK', 'K', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12' ];
var gradeLevels = [
     -1,   0,    1,    2,    3,    4,    5,    6,    7,    8,    9,   10,   11,   12 ];

function addGradeLevel(str, num) {
    gradeStrings.push(str);
    gradeLevels.push(num);
}

function Grade(grades) {
    // Construct student grade level strings.
    // See gradeStrings and gradeLevels.
    // Fix by uppercasing or fiddling with leading zeros.
    // asNumber returns a gradeLevels value, or NaN if it doesn't know the grade designator.
    // See addGradeLevel() to add designators.
    // Require values, by default.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.prefer = 'string';
    this.precision = 2;  // '01', '02', '10'
    this.case = 'upper';
    this.label = 'grade';
    this.validSet = gradeStrings;
    this.validNull = false;
    this.asNumber = 
        function(i) {
            var intExp = /^\d+$/;
            var pos = gradeStrings.indexOf(String(this.v[i]));
            if (pos == -1)
                pos = gradeStrings.indexOf('0' + this.v[i]);
            if (pos == -1)
                return NaN;
            else
                return gradeLevels[pos];
        };
}
Grade.prototype = new EdDatum;
edDataTypes.push(Grade);

var mcasSubjectLabels = [    
    'English', 'math', 'science', 'biology', 'chemistry', 'physics', 'technology/engineering' ];
var mcasSubjectCodes = [
     10,        20,     30,        31,        32,          33,        34 ];

function MCASSubjectCode(codes) {
    // Construct MCAS test subject codes.
    // Require values.
    // asString translates to mcasSubjectLabels value.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.prefer = 'number';
    this.label = 'MCAS subject code';
    this.validNull = false;
    this.validRange.min = 10;
    this.validRange.max = 40;
    this.validRange.integer = true;
    this.asString = function(i) {
        var pos = mcasSubjectCodes.indexOf(Number(this.v[i]));
        if (pos == -1)
            return String(this.v[i]);  // unmatched subject code
        else
            return mcasSubjectLabels[pos];  // subject label
    };
}
MCASSubjectCode.prototype = new EdDatum;
edDataTypes.push(MCASSubjectCode);

function Category(values) {
    // Construct a generic category variable.
    // Missing values are valid, by default.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'category';
}
Category.prototype = new EdDatum;
edDataTypes.push(Category);

function Rating(values) {
    // Construct a text rating variable.
    // Override the order method to assign ordinal numbers to rating values.
    // Missing values are valid, by default.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.prefer = 'string';
    this.label = 'rating';
    this.order = function() { return null };
}
Rating.prototype = new EdDatum;
edDataTypes.push(Rating);

var MCASPerformanceLevels = {
    'WF': { long: 'Warning/Failing',   order: 1,   match: /^wf$|^warn\w*.fail/i   },
    'W':  { long: 'Warning',           order: 1.1, match: /^w$|^warn/i            },
    'F':  { long: 'Failing',           order: 0.9, match: /^f$|^fail/i            },
    'NI': { long: 'Needs Improvement', order: 2,   match: /^ni$|^needs/i          },
    'P':  { long: 'Proficient',        order: 3,   match: /^p$|^prof/i            },
    'A':  { long: 'Advanced',          order: 4,   match: /^a$|^adv/i             },
    'P+': { long: 'Beyond Proficient', order: 4.1, match: /^p\+$|^prof\w* (\+|plus)|^beyond prof/i }
};

function MCASPerformanceLevelRecognizer(s) {
    // Attempt to recognize an MCAS performance level.
    // Return the short name if found, otherwise return null.
    if (s == null) return null;
    for (var i in MCASPerformanceLevels)
        if (MCASPerformanceLevels[i].match.test(s))
            return i;
    return null;
}
edDataTypes.push(MCASPerformanceLevelRecognizer);

function MCASPerformanceLevel(value) {
    // Construct MCAS performance levels.
    // MCAS ALT performance levels include INP, PRG.
    // Other codes slipped into performance level fields are ABS, MED, ELL, PAS,...
    // order(i) returns a number to order performance levels for display.
    Rating.apply(this, arguments);
    this.type = arguments.callee;
    this.order = 
        function(i) {
            var quick = MCASPerformanceLevels[this.v[i]];
            if (quick) return quick.order;
            var shortName = MCASPerformanceLevelRecognizer(this.v[i]);
            if (shortName == null) 
                return null;
            else
                return MCASPerformanceLevels[shortName].order;
        };
}
MCASPerformanceLevel.prototype = new Rating;
edDataTypes.push(MCASPerformanceLevel);

function Score(values) {
    // Construct numeric scores.
    // Set validRange and validNull to constrain or require score values.
    EdDatum.apply(this, arguments);
    this.type = arguments.callee;
    this.prefer = 'number';
    this.label = 'score';
}
Score.prototype = new EdDatum;
edDataTypes.push(Score);

function GradeLevelEquivalentScore(values) {
    Score.apply(this, arguments);
}
GradeLevelEquivalentScore.prototype = new Score;
edDataTypes.push(GradeLevelEquivalentScore);

function PointScore(values) {
    // Construct non-negative, whole-number scores.
    Score.apply(this, arguments);
    this.type = arguments.callee;
    this.validRange.min = 0;
    this.validRange.integer = true;
}
PointScore.prototype = new Score;
edDataTypes.push(PointScore);

function DIBELSScore(pointScores) {
    PointScore.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'DIBELS score';
}
DIBELSScore.prototype = new PointScore;
edDataTypes.push(DIBELSScore);

function SRAScore(pointScores) {
    // Construct fractional scores.
    Score.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'SRA score';
    this.validRange.min = 0;
}
SRAScore.prototype = new Score;
edDataTypes.push(SRAScore);

function MCASRawScore(pointScores) {
    // Construct MCAS raw scores for items, tests, or aggregate totals.
    PointScore.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'raw score';
    this.validRange.min = 0;
}
MCASRawScore.prototype = new PointScore;
edDataTypes.push(MCASRawScore);

function MCASMaximumScore(pointScores) {
    // Construct a maximum points possible variable.
    MCASRawScore.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'maximum score possible';
}
MCASMaximumScore.prototype = new MCASRawScore;
edDataTypes.push(MCASMaximumScore);

function MCASAverageScore(pointScores) {
    // Construct an average raw point score, which may be fractional.
    Score.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'average raw score';
    this.validRange.min = 0;
}
MCASAverageScore.prototype = new Score;
edDataTypes.push(MCASAverageScore);

function MCASScaledScore(scores) {
    // Construct MCAS scaled scores, even values between 200 and 280.
    PointScore.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'scaled score';
    this.validRange.min = 200;
    this.validRange.max = 280;
    this.validRange.integer = true;
}
MCASScaledScore.prototype = new PointScore;
edDataTypes.push(MCASScaledScore);

function MCASStudentCPI(cpiValues) {
    // Construct an MCAS CPI variable.
    // This is the individual student value, which can be 0, 25, 50, 75, or 100, or null.
    PointScore.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'CPI';
    this.validRange.min = 0;
    this.validRange.max = 100;
    this.validRange.integer = true;
    this.validNull = true;
    this.validSet = [0, 25, 50, 75, 100];
}
MCASStudentCPI.prototype = new PointScore;
edDataTypes.push(MCASStudentCPI);

function MCASSGP(values) {
    // Construct student growth percentiles.
    Score.apply(this, arguments);
    this.type = arguments.callee;
    this.label = 'SGP';
    this.validRange.min = 1;
    this.validRange.max = 99;
    this.validRange.integer = true;
}
MCASSGP.prototype = new Score;
edDataTypes.push(MCASSGP);

var dataTypeProductions = [
    [/years/i, Category],
    [/(sims)|(school).*year/i, SchoolYear],
    [/year/i, CalendarYear],
    [/teacher/i, Teacher],
    [/dibels/i, DIBELSScore],
    [/sra/i, SRAScore],
    [/grade/i, Grade],
    [/oct.?1/i, Category],
    [/sasid/i, MASASID],
    [/lasid/i, StudentID],
    [/status/i, Category],
    [/race/i, Category],
    [/gender/i, Category],
    [/alt.*assess/i, Category],
    [/lep|limited english/i, Category],
    [/income|frpl|reduced/i, Category],
    [/504/i, Category],
    [/educat.*environ/i, Category],
    [/(sped)|(special.ed).*eval/i, Category],
    [/sped|special.ed|disab/i, Category],
    [/title.(i|1)/i, Category],
    [/(dist\a*)|(school).?(id)|(code)/i, MAOrgCode],
    [/subj/i, MCASSubjectCode],
    [/max.*score/i, MCASMaximumScore],
    [/raw.?score/i, MCASRawScore],
    [/sgp|growth.perc/i, MCASSGP],
    [/cpi/i, MCASStudentCPI],
    [/scaled.?score|\bs.?s\b/i, MCASScaledScore],
    [/perf.*lev.*/i, MCASPerformanceLevel],
    [/score|point/i, Score],
    [/(first|last).*name/i, Name],
    [/stu.*name/i, Name],
    [/name/i, Name],
    [/.*/, EdDatum]  // guaranteed match
];

function classifyDataName(n) {
    for (var i in dataTypeProductions)
        if (dataTypeProductions[i][0].test(n))
            return dataTypeProductions[i][1];
}

var xpr = {  // regular expressions
    years: /years/i,  // Category
    schoolYear: /(sims)|(school).*year/i,   // SchoolYear
    calendarYear: /year/i,   // CalendarYear
    teacher: /teacher/i,   // Teacher
    dibels: /dibels/i,   // DIBELSScore
    sra: /sra/i,   // SRAScore
    grade: /grade/i,   // Grade
    oct1: /oct.?1/i,   // Category
    sasid: /sasid/i,   // MASASID
    lasid: /lasid/i,   // StudentID
    status: /status/i,   // Category
    race: /race/i,   // Category
    gender: /gender/i,   // Category
    altAssess: /alt.*assess/i,   // Category
    lep: /lep|limited english/i,   // Category
    frpl: /income|frpl|reduced/i,   // Category
    plan504: /504/i,   // Category
    edEnviron: /educat.*environ/i,   // Category
    spedEval: /(sped)|(special.ed).*eval/i,   // Category
    sped: /sped|special.ed|disab/i,   // Category
    title1: /title.(i|1)/i,   // Category
    orgCode: /(dist\a*)|(school).?(id)|(code)/i,   // MAOrgCode
    subject: /subj/i,   // MCASSubjectCode
    maxScore: /max.*score/i,   // MCASMaximumScore
    rawScore: /raw.?score/i,   // MCASRawScore
    sgp: /sgp|growth.perc/i,   // MCASSGP
    cpi: /cpi/i,   // MCASStudentCPI
    scaledScore: /scaled.?score|\bs.?s\b/i,   // MCASScaledScore
    performanceLevel: /perf.*lev.*/i,   // MCASPerformanceLevel
    score: /score|point/i,   // Score
    firstLastName: /(first|last).*name/i,   // Name
    studentName: /stu.*name/i,   // Name
    name: /name/i,   // Name
    anything: /.*/   // EdDatum] -> guaranteed match
};

function classifyVariable(v) {
    var l = v.label;
    if (v.uniqCount == 1 && v.hasMissing) {
        // No values present.
        return EdDatum;
    }
    if (xpr.sasid.test(l)) {
        return MASASID;
    }
    if (xpr.lasid.test(l)) {
        return StudentID;
    }
    if (xpr.years.test(l)) {
        // Plural "years" not to be confused with other year values.
        // Eg, "Years in Massachusetts".
        return Category;
    }
    if (xpr.schoolYear.test(l)) {
        return SchoolYear;
    }
    if (xpr.calendarYear.test(l)) {
        return CalendarYear;
    }
    if (xpr.scaledScore.test(l)) {
        return MCASScaledScore;
    }
    if (xpr.performanceLevel.test(l)) {
        return MCASPerformanceLevel;
    }
    if (xpr.dibels.test(l)) {
        return DIBELSScore;
    }
    if (xpr.sra.test(l)) {
        return SRAScore;
    }
    if (xpr.oct1.test(l) && v.uniqCount < 4) {
        // "enrolled Oct 1"
        return Category;
    }
}
